
# codeing:utf-8


"""
表格预处理脚本::
输入html,tag_ls
输出2个十字数据，业绩十字与公共十字

"""
import re
from lxml import etree
from pandas import DataFrame
import math


class RegexDict():
    def __init__(self,redict,judgekey):
        self.redict = redict
        self.judgekey = judgekey
    def main(self):
        for i in self.redict.keys():
            if re.search(i,self.judgekey):
                return self.redict[i]
        raise KeyError('not in dict')

class Table():
    def __init__(self,**params1):
        self.html_content = ''
        self.end_dict = {}
        self.rubbish_word = params1['rubbish_word']
        self.replace_word = params1['replace_word']
        self.config = params1['config']



    """
        获取首次并列table，输入一个html的全部table标签，返回非嵌套标签（非嵌套的并列标签）
        逻辑思想：遍历每一个table，根据是否存在包含关系，若包含则跳过，否则添加，最后获得的一定是一组完全没有嵌套的首table数据
    """
    def table_clear(self, html_content):
        try:
            #若只有一个则直接返回
            if len(html_content) == 1:
                return html_content
            table_region = [html_content[0]]
            start = etree.tostring(html_content[0])
            for i in html_content[1:]:
                now_str = etree.tostring(i)
                if now_str in start:
                    pass
                else:
                    table_region.append(i)
                    start = now_str
            return table_region
        except Exception as e:
            print('table_clear_wrong:', e)

    """
        获取每个table下的tr与table关系，输入一个html的非嵌套列表，将每个table下的tr组合成一个内层列表，内层嵌套table存为一个内层列表
        逻辑思想：循环每一个首table下的tr，若tr中不包含<table则为此行不含内层嵌套table，直接append，否则内层数据为嵌套table，需要当做table递归处理
    """

    def table_group(self, html_content):
        try:
            out_ls = []
            for i in html_content:
                out_ls_table = []
                tr_list = i.xpath('./tr')
                if tr_list == []:
                    tr_list = i.xpath('./tbody/tr')
                for j in tr_list:
                    if ('<table' not in etree.tostring(j, encoding='utf-8').decode('utf-8')) or \
                            (('<table' in etree.tostring(j, encoding='utf-8').decode('utf-8')) and len(
                                re.findall('<tr', etree.tostring(j, encoding='utf-8').decode('utf-8'))) == 1):
                        out_ls_table.append(j)
                    else:
                        if j.xpath('.//table') == []:
                            continue
                        out_ls_table.append(self.table_group(self.table_clear(j.xpath('.//table'))))
                out_ls.append(out_ls_table)
            return out_ls
        except Exception as e:
            print('table_group_wrong:', e)

    """判断是否存在行合并，输入一个tr的xpath对象，输出是否为行合并以及行合并的行数"""

    def row_concat(self, word):
        temp_columns = word.xpath('@rowspan')

        if (len(temp_columns) == 0) or (temp_columns[0] in ['','undefined']):
            return False, 1
        else:
            if re.search('[/{]',temp_columns[0]) != None:
                temp_columns[0] = 1

            if isinstance(temp_columns[0],str):
                temp_columns[0] = re.sub('@项目组成人数|"|8px|,','',temp_columns[0])
                temp_columns[0] = temp_columns[0].split(';')[0]
            temp_columns[0] = math.floor(float(temp_columns[0]))
            if temp_columns[0] == '':
                return False, 1
            if int(temp_columns[0]) == 1:
                return False,1
            else:
                return True, int(temp_columns[0])

    """判断是否存在列合并，输入一个tr的xpath对象，输出是否为列合并以及行合并的行数"""

    def column_concat(self, word):
        # print(etree.tostring(word,encoding='utf-8').decode('utf-8'))
        temp_columns = word.xpath('@colspan')
        if (len(temp_columns) == 0) or (temp_columns[0] in ['','undefined']):
            return False, 1
        else:
            if re.search('[/{]',temp_columns[0]) != None:
                temp_columns[0] = 1
            if isinstance(temp_columns[0],str):
                temp_columns[0] = re.sub('@项目组成人数|"|8px|,|‘|X|6X<span lang=','',temp_columns[0])
                temp_columns[0] = temp_columns[0].split(';')[0]
            if temp_columns[0] == '':
                return False, 1
            try:
                temp_columns[0] = math.floor(float(temp_columns[0]))
            except:
                return False, 1
            if int(temp_columns[0]) == 1:
                return False, 1
            else:
                return True, int(temp_columns[0])

    """
        拆分合并表格为各个单独的表格，输入为经过table嵌套拆分的数据html列表，输出为拆分过的列表
        逻辑思想：循环每一个tr下的td，根据其colspan判断列合并，存在往后补充，根据其rowspan判断行合并，存在传递信息给下一个td（(位置, [(数据, 0)],传递几行)）
    """


    def concat(self, html_content):
        # try:
        end_table = []
        # 循环每个并列table
        for table in html_content:
            table_temp = []
            later_concat_convey = []
            # 循环每个table下的tr
            for tr in table:
                # 若此tr为列表，则证明这是内嵌表格，使用递归处理
                if isinstance(tr, list):
                    tr_temp = self.concat(tr)
                    table_temp.append(tr_temp)
                else:
                    tr_temp = []
                    # 获取全部td

                    if tr == None:
                        continue
                    td_ls = tr.xpath('.//td')

                    # 将行合并的数据传入合并辐射范围内，later_concat_convey的格式为(该td在tr中的顺序，（合并的td数据，0），合并的行数)
                    index_back = 0
                    if later_concat_convey != []:
                        later_concat_convey_end = []
                        for m, n in enumerate(later_concat_convey):
                            index, td_use, row_num = n
                            if m != 0:
                                index = index + index_back
                            td_ls = td_ls[:index] + td_use + td_ls[index:]
                            row_num -= 1
                            index_back += len(td_use) - 1
                            if row_num != 0:
                                later_concat_convey_end.append((index, td_use, row_num))
                        later_concat_convey = later_concat_convey_end


                    # td_ls是一整行数据，不存在之前的td合并数据，目前的td_ld数据格式为：[<td>,<td>,(<td>,0),(<td>,0)],(<td>,0)是从上级行合并传下来的数据
                    for td in range(len(td_ls)):

                        # 若该td是tuple，则证明为该td是之前行合并传下来的数据，不需要进行处理
                        if isinstance(td_ls[td], tuple):
                            tr_temp.append(td_ls[td][0])
                            continue
                        else:
                            display = td_ls[td].xpath('@style')
                            if display != []:
                                if  'display: none;' in display[0]:
                                    continue
                            # 判断是否列合并，已经列合并的数量
                            key_column_concat, column_concat_num = self.column_concat(td_ls[td])
                            # 将列合并td存入tr_temp,tr_temp是一行的所有数据
                            if key_column_concat == False:
                                tr_temp.append(td_ls[td])
                            else:
                                tr_temp = tr_temp + ([td_ls[td]] * column_concat_num)
                            # 判断是否行合并，若存在行合并则将行合并的数据存入later_concat_convey已传给下层
                            key_row_concat, row_concat_num = self.row_concat(td_ls[td])

                            if key_row_concat == True:
                                if key_column_concat == False:
                                    later_concat_convey.append((td, [(td_ls[td], 0)], row_concat_num - 1))
                                else:
                                    later_concat_convey.append(
                                        (td, [(td_ls[td], 0)] * column_concat_num, row_concat_num - 1))

                    # if later_concat_convey != []:
                    #     print(later_concat_convey,etree.tostring(later_concat_convey[0][1][0][0],encoding = 'utf-8').decode('utf-8'))
                    table_temp.append(tr_temp)

            end_table.append(table_temp)
        return end_table

    # except Exception as e:
    #     print('concat_wrong:', e)

    """将数据从xpath对象取出转化为对应的信息"""

    def word_table(self, html_content, tag_ls):
        def special_key_word(word_temp,spe_re,no_spre_re,left_re,no_left_re,right_re,html_content,table,tr,td,out_show,tag_ls):
            if re.search(spe_re, word_temp) and re.search(no_spre_re, word_temp) == None:
                for lef_td in range(td-1,-1,-1):
                    if html_content[table][tr][lef_td].split('?')[0] not in tag_ls:
                        continue
                    else:
                        if re.search(left_re, html_content[table][tr][lef_td]) and re.search(no_left_re,html_content[table][tr][lef_td]) == None:
                            word_temp = out_show + '?' + word_temp + '_handle'
                            break
                for up_td in range(tr-1,-1,-1):
                    if word_temp[:len(out_show)] != out_show:

                        if td < len(html_content[table][up_td]):
                            if isinstance(html_content[table][up_td][0],list):
                                break
                            if html_content[table][up_td][td].split('?')[0] not in tag_ls:
                                continue
                            else:
                                if re.search(spe_re, word_temp):
                                    if html_content[table][up_td] != []:
                                        if not isinstance(html_content[table][up_td][0], list):
                                            if re.search(left_re, html_content[table][up_td][td]) != None:
                                                word_temp = out_show + '?' + word_temp + '_handle'
                                break
                for right_td in range(td+1,len(html_content[table][tr])):
                    if word_temp[:len(out_show)] != out_show:
                        word_later = html_content[table][tr][right_td].xpath('.//text()')
                        word_temp_later = ''
                        for word_facor_later in word_later:
                            word_temp_later += word_facor_later
                        word_temp_later = re.sub('\s', '', word_temp_later)
                        word_temp_later = self.rubbish_word(word_temp_later)
                        if word_temp_later not in tag_ls:
                            continue
                        else:
                            if re.search(spe_re, word_temp) != None and re.search(right_re, self.replace_word(word_temp_later)) != None:
                                word_temp = out_show + '?' + word_temp + '_handle'
                            break

                for down_td in range(tr + 1,len(html_content[table])):
                    if word_temp[:len(out_show)] != out_show:
                        if html_content[table][down_td] != []:
                            if not isinstance(html_content[table][down_td][0], list) and td < len(html_content[table][down_td]):
                                word_later = html_content[table][down_td][td].xpath('.//text()')
                                word_temp_later = ''
                                for word_facor_later in word_later:
                                    word_temp_later += word_facor_later
                                word_temp_later = re.sub('\s', '', word_temp_later)
                                word_temp_later = self.rubbish_word(word_temp_later)
                                if word_temp_later not in tag_ls:
                                    continue
                                else:
                                    if re.search(spe_re, word_temp) != None and re.search(right_re, self.replace_word(word_temp_later)) != None:
                                        word_temp = out_show + '?' + word_temp + '_handle'
                                    break

            return word_temp
        for table in range(len(html_content)):
            for tr in range(len(html_content[table])):
                if html_content[table][tr] == []:
                    continue
                if not isinstance(html_content[table][tr][0], list):
                    for td in range(len(html_content[table][tr])):
                        word = html_content[table][tr][td].xpath('.//text()')
                        word_temp = ''
                        for word_facor in word:
                            word_temp += word_facor
                        word_temp = re.sub('\s', '', word_temp)
                        # word_temp = word_temp.strip()
                        # word_temp = re.sub('\s+',' ',word_temp)

                        word_temp = self.rubbish_word(word_temp)

                        word_temp = special_key_word(word_temp, '^姓名：?$', '-1', '|', '^职[务业]?', '-1', html_content,
                                         table, tr, td, '负责人', tag_ls)
                        word_temp = special_key_word(word_temp, '^(企业|单位)?名称：?$', '负责人|项目经理', '^排名?|^中标公司', '-1', '中标金额', html_content,
                                         table, tr, td, '中标公司', tag_ls)

                        gu = ['业主','招标代理单位']
                        word_temp = special_key_word(word_temp, '^名称：?$', '-1', '(采购|项目)?(业主|比[优选]|询价|询比|招标(（(采购|建设)）)?|建设|主办|[出招]租|出让|原?采购|发包|合同甲)([名全]称|单位|公司|部门|信息|机构|[为：、/（人方]){1,}|^业主$', '-1', '-1', html_content,
                                         table, tr, td, '业主名称', tag_ls)
                        word_temp = special_key_word(word_temp, '^名称：?$', '-1','(招标|交易|采购)代理(机构|公司|单位|系统|名称|人)|^(招标代理|代理机构|填报单位)$','-1', '-1', html_content,
                                                     table, tr, td, '招标代理单位名称', tag_ls)
                        for gui in gu:
                            word_temp = special_key_word(word_temp,'^地址：?$','-1','^'+gui+'(名称|联系(人|电话)|传真)','-1','-1', html_content,
                                             table, tr, td, gui+'地址', tag_ls)
                            word_temp = special_key_word(word_temp,'^联系人：?$','-1','^'+gui+'(名称|联系电话|地址|传真)','-1','-1', html_content,
                                             table, tr, td, gui+'联系人', tag_ls)
                            word_temp = special_key_word(word_temp,'^((联系人?)?电话(（传真）)?|联系方式)：?$','-1','^'+gui+'(名称|联系人|地址|传真)','-1','-1', html_content,
                                             table, tr, td, gui+'联系电话', tag_ls)
                            word_temp = special_key_word(word_temp,'^传真：?$','-1','^'+gui+'(名称|联系人|地址|联系电话)','-1','-1', html_content,
                                             table, tr, td, gui+'传真', tag_ls)
                        if '_handle' in word_temp:
                            word_temp = word_temp.split('_handle')[0]

                        elif re.search('(中标公司|业主地址)\?',word_temp) == None:
                            word_temp = self.replace_word(word_temp).replace('?', '？') + '?' + word_temp.replace('?', '？')
                        html_content[table][tr][td] = word_temp
                else:
                    html_content[table][tr] = self.word_table(html_content[table][tr], tag_ls)

        return html_content



    """获取table中每个单元格的数据信息-主函数，传入的是一个二维dataframe，包括：
    {'单元格数据':
        {
            x:'0'，x坐标
            y:'0'，y坐标
            x_data:[1,2,3],此横行数据
            y_data:[1,2,3],此纵行数据
            use_key:True,是否是配置文件
            direction:right，数据索引方向  
        }
    }
    """

    def get_data_dict(self, now_dataframe_handle, tag_ls,need_keys):

        data_dict = {}
        # 每行
        for index_num, index in enumerate(list(now_dataframe_handle.index)):
            # 每列
            for column_num, column in enumerate(list(now_dataframe_handle.columns)):
                # 每个关键词
                key_word,original_word = now_dataframe_handle[column][index].split('?')
                # key_word = mongobd_word(key_word)

                if re.search('^(成交|评审|中标)结果：?$', key_word) != None:


                    x_data_company = list(now_dataframe_handle.iloc[index_num])
                    y_data_company = list(now_dataframe_handle[column])
                    company_key = ''
                    money_key = False
                    for i in x_data_company:
                        if i[:4] == '中标金额':
                            money_key = True
                            break
                        elif i[:4] == '中标公司':
                            company_key = i
                    if money_key == False and company_key != '':
                        key_word = '中标金额'

                    if key_word != '中标金额':
                        company_key = ''
                        money_key = False
                        for i in y_data_company:
                            if i[:4] == '中标金额':
                                money_key = True
                                break
                            elif i[:4] == '中标公司':
                                company_key = i
                        if money_key == False and company_key != '':
                            key_word = '中标金额'

                key_word = key_word + '{' + str(column_num) + '_' + str(index_num) + '}'
                data_dict[key_word] = {}
                data_dict[key_word]['x'] = column_num
                data_dict[key_word]['y'] = index_num
                data_dict[key_word]['x_data'] = [i.split('?')[0] for i in list(now_dataframe_handle.iloc[index_num])]
                if index_num >= 30:
                    data_dict[key_word]['y_data'] = list(now_dataframe_handle[column])[index_num-30:index_num+30]
                else:
                    data_dict[key_word]['y_data'] = list(now_dataframe_handle[column])[:index_num+30]
                data_dict_gender = (i.split('?')[0] for i in data_dict[key_word]['y_data'])
                data_dict[key_word]['y_data'] = []
                for y_data_i in data_dict_gender:
                    data_dict[key_word]['y_data'].append(y_data_i)

                data_dict[key_word]['original'] = original_word
                for _ in need_keys.keys():
                    if re.search(_,key_word.split('{')[0]) and re.search('[:：]',key_word.split('{')[0]) == None:
                        data_dict[key_word]['use_key'] = True
                        break
                    else:
                        data_dict[key_word]['use_key'] = False

                if data_dict[key_word]['use_key'] == False:

                    if (key_word[:5] == '中标公司_' or key_word[:4] == '中标金额' or key_word[:5] == '中标公司-') and ('^项目名称$' not in need_keys) and re.search('[:：]',key_word) == None:
                        data_dict[key_word]['use_key'] = True
                # 判断方向
                if data_dict[key_word]['use_key'] == True:

                    judge_value_x = 'null'
                    judge_value_y = 'null'
                    if column_num != len(now_dataframe_handle.columns) - 1:
                        judge_value_x = now_dataframe_handle[list(now_dataframe_handle.columns)[column_num + 1]][index].split('?')[0]
                    if index_num != len(now_dataframe_handle.index) - 1:
                        judge_value_y = now_dataframe_handle[column][list(now_dataframe_handle.index)[index_num + 1]].split('?')[0]
                    if judge_value_x == '':
                        judge_value_x = '/'
                    if judge_value_y == '':
                        judge_value_y = '/'
                    if key_word.split('{')[0] in (judge_value_x,judge_value_y):
                        data_dict[key_word]['direction'] = 'None'
                        continue
                    if judge_value_x != 'null':
                        if judge_value_y in tag_ls or judge_value_y == 'rubbish_word_tag':
                            if judge_value_x not in tag_ls:
                                data_dict[key_word]['direction'] = 'right'
                                continue
                        else:
                            if judge_value_x not in tag_ls and judge_value_x != 'rubbish_word_tag':
                                if judge_value_x == '/':
                                    if column_num != 0:
                                        if now_dataframe_handle.iloc[index_num][column_num-1] in tag_ls:
                                            data_dict[key_word]['direction'] = 'down'
                                        else:
                                            data_dict[key_word]['direction'] = 'right'
                                    else:
                                        data_dict[key_word]['direction'] = 'right'

                                elif re.search(RegexDict(need_keys,key_word.split('{')[0]).main(), judge_value_x) != None:

                                    data_dict[key_word]['direction'] = 'right'
                                    continue
                    if judge_value_y != 'null':
                        if judge_value_x in tag_ls or judge_value_x == 'rubbish_word_tag':
                            if judge_value_y not in tag_ls:
                                data_dict[key_word]['direction'] = 'down'
                                continue
                        else:
                            if judge_value_y == '/':
                                if index_num != 0:
                                    if now_dataframe_handle.iloc[index_num-1][column_num] in tag_ls:

                                        data_dict[key_word]['direction'] = 'right'
                                    else:
                                        data_dict[key_word]['direction'] = 'down'
                                else:
                                    data_dict[key_word]['direction'] = 'down'
                            elif judge_value_y not in tag_ls and judge_value_y != 'judge_value_y':
                                if re.search(RegexDict(need_keys,key_word.split('{')[0]).main(), judge_value_y) != None:
                                    data_dict[key_word]['direction'] = 'down'
                                    continue
                    if judge_value_x == 'null':
                        if (judge_value_y not in tag_ls) or (
                                judge_value_y in tag_ls and re.search('中标公司_第', judge_value_y) != None and key_word[:2] == '排名'):
                            data_dict[key_word]['direction'] = 'down'
                            continue
                    if judge_value_y == 'null':
                        if (judge_value_x not in tag_ls) or (judge_value_x in tag_ls and re.search('中标公司_第', judge_value_x) != None and key_word[:2] == '排名'):
                            data_dict[key_word]['direction'] = 'right'
                            continue
                    data_dict[key_word]['direction'] = 'None'
                else:
                    data_dict[key_word]['direction'] = 'None'
        return data_dict
    """获取table中每个单元格的数据信息-表格嵌套函数，传入的是一个二维多维列表"""

    def data_table(self, html_content, tag_ls,need_keys):
        def handle_concat(now_dataframe,data_dict):
            max_num = 0
            for i in now_dataframe:
                max_num = max(max_num, len(i))
            # 二维列表封装为dataframe，fillna为/
            now_dataframe_handle = DataFrame(now_dataframe, columns=['x' + str(i) for i in range(max_num)],
                                             index=['y' + str(i) for i in range(len(now_dataframe))])
            now_dataframe_handle = now_dataframe_handle.fillna('/?/')

            having_data = data_dict['table' + str(table)].keys()
            having_data_ls = []
            # 开始处理多级嵌套的关系，data0，table0......
            for having_data_num in having_data:
                if 'data' in having_data_num:
                    having_data_ls.append(int(having_data_num[4:]))
            if having_data_ls == []:
                data_num = 'data0'
            else:
                tt_data_num = max(having_data_ls) + 1
                data_num = 'data' + str(tt_data_num)
            tt_ls = list(now_dataframe_handle.values.flatten())
            tt_key = False

            data_dict['table' + str(table)][data_num] = self.get_data_dict(now_dataframe_handle, tag_ls,need_keys)
            return data_dict,having_data
        data_dict = {}

        # 循环每个table
        for table in range(len(html_content)):

            data_dict['table' + str(table)] = {}
            now_dataframe = []
            # 循环每一行
            for tr in range(len(html_content[table])):
                # 如果tr[0]为列表，则这为嵌套表格
                if html_content[table][tr] == []:
                    continue

                if not isinstance(html_content[table][tr][0],list):
                    now_dataframe.append(html_content[table][tr])

                else:

                    if now_dataframe != []:

                        """对之前的tr进行处理并对该嵌套表格进行递归处理"""
                        # 获取每个tr下最长数量（防止有的地方会是空）
                        data_dict,having_data = handle_concat(now_dataframe, data_dict)
                        now_dataframe = []
                        having_table_ls = []
                        for having_table_num in having_data:
                            if 'table' in having_table_num:
                                having_table_ls.append(int(having_table_num[5:]))
                        if having_table_ls == []:
                            table_num = 'table0'
                        else:
                            tt_table_num = max(having_table_ls) + 1
                            table_num = 'table' + str(tt_table_num)
                        data_dict['table' + str(table)][table_num] = self.data_table(html_content[table][tr], tag_ls,need_keys)
                        # now_dataframe_handle.to_csv('111.csv',mode = 'a')
                    else:
                        """对该嵌套表格进行递归处理"""
                        having_data = data_dict['table' + str(table)].keys()
                        having_table_ls = []
                        for having_table_num in having_data:
                            if 'table' in having_table_num:
                                having_table_ls.append(int(having_table_num[5:]))
                        if having_table_ls == []:
                            table_num = 'table0'
                        else:
                            tt_table_num = max(having_table_ls) + 1
                            table_num = 'table' + str(tt_table_num)
                        data_dict['table' + str(table)][table_num] = self.data_table(html_content[table][tr], tag_ls,need_keys)
                if tr == (len(html_content[table]) - 1):

                    # self.main_handle(now_dataframe,end_dict,table)
                    """对之前的tr进行处理并对该嵌套表格进行递归处理"""
                    data_dict,having_data = handle_concat(now_dataframe, data_dict)
                    now_dataframe = []

        return data_dict



    def main(self, html,tag_ls,database):

        self.html_content = html
        if database == True:
            self.html_content = html
        else:
            self.html_content = open(html, encoding='utf-8').read()
        if '<table' not in self.html_content and '<TABLE' not in self.html_content:
            return 'no table', ''

        tt = len(re.findall('class=".{0,10}foot', self.html_content))
        self.html_conten = ''
        if tt == 1:
            self.html_conten = re.split('class=".{0,10}foot', self.html_content)[0]
        else:
            for _ in re.split('class=".{0,10}foot', self.html_content):
                if re.search('((((中|入|竞|受|合|承)(标|包|签|选|接|围|让|同)|成交|签约|竞得|施工|推荐|服务)((候|侯)选)?(人|供应(商)?|单位|企业|公司|商|银行|联合体|方|机构))|((买受)?(第一中标|候选人|生产商|报价人|供应商|厂家|单位名称|公司名称|企业名称|响应人|买受人|投人|联合体|设计方|合同乙方)))(全称|信息|如下|成员|方|主办方|单位|牵头人)?([:：为*])?',_) and 'table' in _:
                    self.html_conten = _
                    break
        if self.html_conten == '':
            self.html_conten = re.split('class=".{0,10}foot', self.html_content)[0]
        self.html_conten = re.sub('</?thead.*?>|</?tbody.*?>', '', self.html_conten)
        self.html_conten = re.sub('<th|<TH', '<td', self.html_conten)
        self.html_conten = re.sub('</th>|</TH>', '</td>', self.html_conten)
        self.html_conten = re.sub('\\\\', '/', self.html_conten)
        if re.search('[唱开]标(记录|情况)',self.html_conten):
            record_key = True
        else:
            record_key = False
        try:
            self.html_conten = etree.HTML(self.html_conten)
        except:
            self.html_conten = etree.HTML(self.html_conten.encode('utf-8'))
        self.html_content = self.html_conten.xpath('//table')
        if self.html_content == []:
            return 'no table',''

        self.html_content = self.table_clear(self.html_content)
        self.html_content = self.table_group(self.html_content)
        self.html_content = self.concat(self.html_content)
        html_content = self.word_table(self.html_content, tag_ls)
        data_dict_partone = self.data_table(html_content,tag_ls,self.config['need_keys_partone'])
        return data_dict_partone,record_key



# 处理table类


def data_key_ls(data_dict):
    data_key_ls_end = []
    for i in data_dict.keys():
        if (i[:5] == 'table') or (i[:4] == 'data'):
            data_key_ls_end = data_key_ls_end + data_key_ls(data_dict[i])
        else:
            data_key = i.split('{')[0]
            if len(data_key) <= 10:
                data_key_ls_end.append(data_key)
            else:
                pass

    return data_key_ls_end





